{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <center>Hyperparameter Tuning for Conditional GAN (NLS-KDD)<center/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import defaultdict\n",
    "import pickle, os, itertools\n",
    "\n",
    "import utils , preprocessing  \n",
    "\n",
    "import tensorflow as tf\n",
    "from keras.layers import Dense, Input, Dropout,concatenate\n",
    "from keras.models import Model\n",
    "from keras import backend as K\n",
    "from scipy.stats import norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install tensorflow==1.13.0rc2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check for GPU Availability"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[name: \"/device:CPU:0\"\n",
      "device_type: \"CPU\"\n",
      "memory_limit: 268435456\n",
      "locality {\n",
      "}\n",
      "incarnation: 1036361721152939545\n",
      "]\n"
     ]
    }
   ],
   "source": [
    "from tensorflow.python.client import device_lib\n",
    "print(device_lib.list_local_devices())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Conditional Generative Adversarial Network Class definition"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CGAN():\n",
    "    \"\"\"Conditinal Generative Adversarial Network class\"\"\"\n",
    "    \n",
    "    def __init__(self,arguments,X,y):\n",
    "        [self.rand_noise_dim, self.tot_epochs, self.batch_size,self.D_epochs, \\\n",
    "         self.G_epochs,self.learning_rate, self.n_layers, self.activation,self.optimizer, self.min_num_neurones] = arguments\n",
    "\n",
    "        self.X_train = X\n",
    "        self.y_train = y\n",
    "\n",
    "        self.label_dim = y.shape[1]\n",
    "        self.x_data_dim = X.shape[1]\n",
    "\n",
    "        self.g_losses = []\n",
    "        self.d_losses, self.disc_loss_real, self.disc_loss_generated = [], [], []\n",
    "        self.acc_history = []\n",
    "        \n",
    "        self.__define_models()\n",
    "        self.gan_name = '_'.join(str(e) for e in arguments).replace(\".\",\"\")\n",
    "        \n",
    "        self.terminated = False\n",
    "\n",
    "    def build_generator(self,x,labels):\n",
    "        \"\"\"Create the generator model G(z,l) : z -> random noise , l -> label (condition)\"\"\"\n",
    "        \n",
    "        x = concatenate([x,labels])\n",
    "        for i in range(1,self.n_layers+1):\n",
    "            x = Dense(self.min_num_neurones*i, activation=self.activation)(x)\n",
    "            \n",
    "        x = Dense(self.x_data_dim)(x)\n",
    "        x = concatenate([x,labels])\n",
    "\n",
    "        return x\n",
    "\n",
    "    def build_discriminator(self,x):\n",
    "        \"\"\"Create the discrimnator model D(G(z,l)) : z -> random noise , l -> label (condition)\"\"\"\n",
    "        \n",
    "        for n in reversed(range(1,self.n_layers+1)):\n",
    "            x = Dense(self.min_num_neurones*n, activation=self.activation)(x)\n",
    "        \n",
    "        x = Dense(1, activation='sigmoid')(x)\n",
    "\n",
    "        return x\n",
    "\n",
    "    def __define_models(self):\n",
    "        \"\"\"Define Generator, Discriminator & combined model\"\"\"\n",
    "        \n",
    "        # Create & Compile generator\n",
    "        generator_input = Input(shape=(self.rand_noise_dim,))\n",
    "        labels_tensor = Input(shape=(self.label_dim,))\n",
    "        generator_output = self.build_generator(generator_input, labels_tensor)\n",
    "\n",
    "        self.generator = Model(inputs=[generator_input, labels_tensor], outputs=[generator_output], name='generator')\n",
    "        self.generator.compile(loss='binary_crossentropy',optimizer=self.optimizer, metrics=['accuracy'])\n",
    "        K.set_value(self.generator.optimizer.lr,self.learning_rate)\n",
    "        \n",
    "\n",
    "        # Create & Compile generator\n",
    "        discriminator_model_input = Input(shape=(self.x_data_dim + self.label_dim,))\n",
    "        discriminator_output = self.build_discriminator(discriminator_model_input)\n",
    "\n",
    "        self.discriminator = Model(inputs=[discriminator_model_input],outputs=[discriminator_output],name='discriminator')\n",
    "        self.discriminator.compile(loss='binary_crossentropy',optimizer=self.optimizer, metrics=['accuracy'])\n",
    "        K.set_value(self.discriminator.optimizer.lr,self.learning_rate)\n",
    "\n",
    "        # Build \"frozen discriminator\"\n",
    "        frozen_discriminator = Model(inputs=[discriminator_model_input],outputs=[discriminator_output],name='frozen_discriminator')\n",
    "        frozen_discriminator.trainable = False\n",
    "\n",
    "        # Debug 1/3: discriminator weights\n",
    "        n_disc_trainable = len(self.discriminator.trainable_weights)\n",
    "\n",
    "        # Debug 2/3: generator weights\n",
    "        n_gen_trainable = len(self.generator.trainable_weights)\n",
    "\n",
    "        # Build & compile combined model from frozen weights discriminator\n",
    "        combined_output = frozen_discriminator(generator_output)\n",
    "        self.combined = Model(inputs = [generator_input, labels_tensor],outputs = [combined_output],name='adversarial_model')\n",
    "        self.combined.compile(loss='binary_crossentropy',optimizer=self.optimizer, metrics=['accuracy'])\n",
    "        K.set_value(self.combined.optimizer.lr,self.learning_rate)\n",
    "\n",
    "        # Debug 3/3: compare if trainable weights correct\n",
    "        assert(len(self.discriminator._collected_trainable_weights) == n_disc_trainable)\n",
    "        assert(len(self.combined._collected_trainable_weights) == n_gen_trainable)\n",
    "        \n",
    "    def __get_batch_idx(self):\n",
    "        \"\"\"random selects batch_size samples indeces from training data\"\"\"\n",
    "        \n",
    "        batch_ix = np.random.choice(len(self.X_train), size=self.batch_size, replace=False)\n",
    "\n",
    "        return batch_ix\n",
    "    \n",
    "    def dump_to_file(self,save_dir = \"./logs\"):\n",
    "        \"\"\"Dumps the training history and GAN config to pickle file \"\"\"\n",
    "        \n",
    "        H = defaultdict(dict)\n",
    "        H[\"acc_history\"] = self.acc_history\n",
    "        H[\"Generator_loss\"] = self.g_losses\n",
    "        H[\"disc_loss_real\"] = self.disc_loss_real\n",
    "        H[\"disc_loss_gen\"] = self.disc_loss_generated\n",
    "        H[\"discriminator_loss\"] = self.d_losses\n",
    "        H[\"rand_noise_dim\"] , H[\"total_epochs\"] = self.rand_noise_dim, self.tot_epochs\n",
    "        H[\"batch_size\"] , H[\"learning_rate\"]  = self.batch_size, self.learning_rate\n",
    "        H[\"n_layers\"] , H[\"activation\"]  = self.n_layers, self.activation\n",
    "        H[\"optimizer\"] , H[\"min_num_neurones\"] = self.optimizer, self.min_num_neurones\n",
    "        \n",
    "        if not os.path.exists(save_dir):\n",
    "            os.makedirs(save_dir)\n",
    "        \n",
    "        with open(f\"{save_dir}/{self.gan_name}{'.pickle'}\", \"wb\") as output_file:\n",
    "            pickle.dump(H,output_file)\n",
    "        \n",
    "    def train(self):\n",
    "        \"\"\"Trains the CGAN model\"\"\"\n",
    "\n",
    "        # Adversarial ground truths\n",
    "        real_labels = np.ones((self.batch_size, 1))\n",
    "        fake_labels = np.zeros((self.batch_size, 1))\n",
    "        # Adversarial ground truths with noise\n",
    "        #real_labels = np.random.uniform(low=0.999, high=1.0, size=(self.batch_size,1))\n",
    "        #fake_labes = np.random.uniform(low=0, high=0.00001, size=(self.batch_size,1))\n",
    "\n",
    "        for epoch in range(self.tot_epochs):\n",
    "            #Train Discriminator\n",
    "            for i in range(self.D_epochs):\n",
    "\n",
    "                idx = self.__get_batch_idx()\n",
    "                x, labels = self.X_train[idx], self.y_train[idx]\n",
    "\n",
    "                #Sample noise as generator input\n",
    "                noise = np.random.normal(0, 1, (self.batch_size, self.rand_noise_dim))\n",
    "\n",
    "                #Generate a half batch of new images\n",
    "                generated_x = self.generator.predict([noise, labels])\n",
    "\n",
    "                #Train the discriminator\n",
    "                d_loss_fake = self.discriminator.train_on_batch(generated_x, fake_labels)\n",
    "                d_loss_real = self.discriminator.train_on_batch(np.concatenate((x,labels),axis=1), real_labels)\n",
    "                d_loss = 0.5 * np.add(d_loss_real,d_loss_fake)\n",
    "\n",
    "            self.disc_loss_real.append(d_loss_real[0])\n",
    "            self.disc_loss_generated.append(d_loss_fake[0])\n",
    "            self.d_losses.append(d_loss[0])\n",
    "            self.acc_history.append([d_loss_fake[1],d_loss_real[1]])\n",
    "            \n",
    "            #NB: Gradients could be exploding or vanishing (cliping or changing activation function,learning rate could be a solution)\n",
    "            if np.isnan(d_loss_real) or np.isnan(d_loss_fake):\n",
    "                self.terminated = True\n",
    "                break\n",
    "                \n",
    "            #Train Generator (generator in combined model is trainable while discrimnator is frozen)\n",
    "            for j in range(self.G_epochs):\n",
    "                #Condition on labels\n",
    "                # sampled_labels = np.random.randint(1, 5, self.batch_size).reshape(-1, 1)\n",
    "                sampled_labels = np.random.choice([0,2,3,4],(self.batch_size,1), replace=True)\n",
    "\n",
    "                #Train the generator\n",
    "                g_loss = self.combined.train_on_batch([noise, sampled_labels], real_labels)\n",
    "                self.g_losses.append(g_loss[0])\n",
    "\n",
    "            #Print metrices\n",
    "            print (\"Epoch : {:d} [D loss: {:.4f}, acc.: {:.4f}] [G loss: {:.4f}]\".format(epoch, d_loss[0], 100*d_loss[1], g_loss[0]))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read & Preprocess Input data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train,test, label_mapping = preprocessing.get_data(encoding=\"Label\")\n",
    "data_cols = list(train.columns[ train.columns != 'label' ])\n",
    "x_train , x_test = preprocessing.preprocess(train,test,data_cols,\"Robust\",True)\n",
    "\n",
    "y_train = x_train.label.values\n",
    "y_test = x_test.label.values\n",
    "\n",
    "data_cols = list(x_train.columns[ x_train.columns != 'label' ])\n",
    "\n",
    "to_drop = preprocessing.get_contant_featues(x_train,data_cols)\n",
    "x_train.drop(to_drop, axis=1,inplace=True)\n",
    "x_test.drop(to_drop, axis=1,inplace=True)\n",
    "\n",
    "data_cols = list(x_train.columns[ x_train.columns != 'label' ])\n",
    "\n",
    "att_ind = np.where(x_train.label != label_mapping[\"normal\"])[0]\n",
    "x = x_train[data_cols].values[att_ind]\n",
    "y = y_train[att_ind].reshape(-1,1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Set parameters to tune"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# rand_dim = np.arange(10,110,10)\n",
    "base_n_count = np.arange(3,41,3)\n",
    "ephocs = np.arange(100,5000,100)\n",
    "batch_sizes = [64,128,250,300,350]\n",
    "learning_rates = np.logspace(-1,-4,num=20)\n",
    "num_layers = np.arange(3,20)\n",
    "\n",
    "optimizers = [\"sgd\", \"RMSprop\", \"adam\", \"Adagrad\", \"Adamax\",\"Nadam\"]\n",
    "activation_func = [\"tanh\",\"relu\",\"softplus\",\"linear\",\"elu\"]\n",
    "\n",
    "#create a logs directory\n",
    "if not os.path.exists('logs'):\n",
    "    os.makedirs('logs')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Using itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "MemoryError",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-6-4c98826b56c6>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m      1\u001b[0m tot = list(itertools.product([32],ephocs,batch_sizes,[1],[1],\\\n\u001b[1;32m----> 2\u001b[1;33m                              learning_rates,num_layers,activation_func,optimizers,base_n_count))\n\u001b[0m\u001b[0;32m      3\u001b[0m \u001b[0mk\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtot\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m     \u001b[0margs\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mi\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mMemoryError\u001b[0m: "
     ]
    }
   ],
   "source": [
    "tot = list(itertools.product([32],ephocs,batch_sizes,[1],[1],\\\n",
    "                             learning_rates,num_layers,activation_func,optimizers,base_n_count))\n",
    "for i in tot:\n",
    "    args = list(i)\n",
    "    cgan = CGAN(args,x,y)\n",
    "    cgan.train()\n",
    "    if not cgan.terminated :\n",
    "        cgan.dump_to_file()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Grid Search loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ep_d = 1\n",
    "# ep_g = 1\n",
    "\n",
    "# for ep in ephocs:\n",
    "#     for lr in learning_rates:\n",
    "#         for op in optimizers:\n",
    "#             for ac in activation_func:\n",
    "#                 for n_layers in num_layers:\n",
    "#                     for base_n in base_n_count:\n",
    "#                         for BS in batch_sizes:\n",
    "#                             args = [32, ep, BS, ep_d ,ep_g, lr, n_layers, ac ,op, base_n]\n",
    "#                             cgan = CGAN(args,x,y)\n",
    "#                             cgan.train()\n",
    "#                             cgan.dump_to_file()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# args = [32, 500, 130,1 ,1, 0.001, 4, \"tanh\" ,\"sgd\", 50]\n",
    "# cgan = CGAN(args,x,y.reshape(-1,1))\n",
    "# cgan.train()\n",
    "# cgan.dump_to_file()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
